package org.liuh.designs.spider;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import lombok.AllArgsConstructor;
import lombok.Getter;
import org.liuh.designs.spider.dtos.AddressJsonNode;
import org.liuh.designs.spider.dtos.AddressNode;
import org.liuh.designs.spider.utils.HttpClientUtil;
import org.liuh.designs.spider.utils.NioFileReader;
import org.liuh.designs.spider.utils.ReaderListener;
import org.liuh.designs.spider.utils.RedisUtils;
import redis.clients.jedis.Jedis;

import java.awt.geom.Area;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;

/**
 * @ClassName: CountySpiderMain
 * @Description: 街道爬取..
 * @Author: liuh
 * @Date: 2020/10/12 15:17
 * @Version: 1.0
 */
public class TownSpiderMain {

    public final static StringBuilder sourcePath = new StringBuilder("C:\\Users\\Administrator\\Desktop\\2020\\任务\\地址库\\data\\");

    public static RedisUtils redisUtils = new RedisUtils();

    /**
     * @Description: 测试单条.. 没问题..
     * @Author: liuh
     * @Date: 2020/10/12
     */
    public static void main(String[] args) {
        //TODO 测试
        /* testSingleSpiderCounty() */
        HttpClientUtil.resetProxyIP(200, 200);
        spiderTownMethod();
    }

    /**
     * @Description: 爬取城镇信息..
     * @Author: liuh
     * @Date: 2020/10/12
     */
    public static void spiderTownMethod() {
        try {
            long startSpiderTime = System.currentTimeMillis();
            List<AddressNode> countryNodeList = getNeedSpiderCounty();
            if (countryNodeList == null || countryNodeList.isEmpty()) {
                return;
            }
            System.out.println(String.format("获取待处理区县数量： %s", countryNodeList.size()));
            List<AddressNode> totalTownList = doSpiderTown(countryNodeList);
            long endSpiderTime = System.currentTimeMillis();
            System.out.println(String.format("获取所有街道信息完成， 街道总数： %s, 耗时： %s", totalTownList.size(), (endSpiderTime - startSpiderTime) / 1000));
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }

    /**
     * @Description: 爬取
     * @Author: liuh
     * @Date: 2020/10/13
     */
    private static List<AddressNode> doSpiderTown(List<AddressNode> countryNodeList) throws Exception {
        ThreadPoolManager poolManager = ThreadPoolManager.getInstance();
        ExecutorService threadPool = poolManager.getCachedThreadPool();
        CountDownLatch countyDown = new CountDownLatch(countryNodeList.size());
        CopyOnWriteArrayList<AddressNode> waitDealList = new CopyOnWriteArrayList<>();
        for (AddressNode addressNode : countryNodeList) {
            AreaSpiderCountyTask countyTask = new AreaSpiderCountyTask(addressNode, countyDown, AreaConstant.HttpUtilEnum.HTTP_CLIENT, waitDealList);
            threadPool.execute(countyTask);
        }
        countyDown.await();
        threadPool.shutdown();
//            System.out.println(String.format("未获取到街道信息区县数量： %s", waitDealList.size()));
        // TODO 后期引入重试机制..
           /* CopyOnWriteArrayList<AddressNode> againWaitDealList = new CopyOnWriteArrayList<>();
            CountDownLatch againtest = new CountDownLatch(waitDealList.size());
            for (AddressNode addressNode : waitDealList) {
                AreaSpiderCountyTask countyTask = new AreaSpiderCountyTask(addressNode, againtest, AreaConstant.HttpUtilEnum.HTTP_CLIENT, againWaitDealList);
                threadPool.execute(countyTask);
            }
            againtest.await();
            System.out.println(String.format("未获取到街道信息区县数量： %s", againWaitDealList.size()));*/
        List<AddressNode> totalTownList = new ArrayList<>();
        for (AddressNode addressNode : countryNodeList) {
            if (addressNode.getChildNodeList() != null) {
                totalTownList.addAll(addressNode.getChildNodeList());
            }
        }
        System.out.println();
        System.out.println("------------------------所有街道信息开始-------------------------");
        JSONArray townJSONArray = new JSONArray();
        JSONObject objTown;
        for (AddressNode townNode : totalTownList) {
            objTown = new JSONObject();
            objTown.put("code", townNode.getAddressCode());
            objTown.put("name", townNode.getAddressName());
            objTown.put("url", townNode.getNodeUrl());
            if (townNode.getParentNode() != null) {
                objTown.put("countyCode", townNode.getParentNode().getAddressCode());
                objTown.put("countyName", townNode.getParentNode().getAddressName());
                if (townNode.getParentNode().getParentNode() != null) {
                    objTown.put("cityCode", townNode.getParentNode().getParentNode().getAddressCode());
                    objTown.put("cityName", townNode.getParentNode().getParentNode().getAddressName());
                    AddressNode provinceNode = townNode.getParentNode().getParentNode().getParentNode();
                    if (provinceNode != null) {
                        objTown.put("provinceCode", provinceNode.getAddressCode());
                        objTown.put("provinceName", provinceNode.getAddressName());
                    }
                }
            }
            townJSONArray.add(objTown);
        }
        redisUtils.setnx(AreaConstant.RDS_TOWN_KEY, JSON.toJSONString(townJSONArray));
        System.out.println();
        // 未处理集合信息..
        JSONArray waitDealArray = new JSONArray();
        for (AddressNode waitNode : waitDealList) {
            waitDealArray.add(waitNode);
        }
        redisUtils.setnx(AreaConstant.getWaitKey(), JSON.toJSONString(waitDealArray));
        return totalTownList;
    }

    /**
     * @Description: 获取需要处理的区县信息..
     * @Author: liuh
     * @Date: 2020/10/13
     */
    private static List<AddressNode> getNeedSpiderCounty() throws Exception {
        // TODO 1. 获取区县信息..
        String countryData = getRdsDataByKey(AreaConstant.RDS_COUNTY_KEY);
        if (countryData == null) {
            return null;
        }
        List<AddressJsonNode> countryList = JSON.parseArray(countryData, AddressJsonNode.class);
        if (countryList == null || countryList.isEmpty()) {
            return null;
        }
        // TODO 2. 加载市. 给他安排父节点请求的url.
        String cityData = getRdsDataByKey(AreaConstant.RDS_CITY_KEY);
        if (cityData == null) {
            return null;
        }
        List<AddressJsonNode> cityList = JSON.parseArray(cityData, AddressJsonNode.class);
        if (cityList == null || cityList.isEmpty()) {
            return null;
        }
        // 设置sourceUrl..
        for (AddressJsonNode cityNode : cityList) {
            Optional<AddressJsonNode> first = countryList.stream().filter(c -> c.getCityCode().equals(cityNode.getCode())).findFirst();
            if (first.isPresent() && first.get() != null) {
                AddressJsonNode countryNode = first.get();
                countryNode.setParentUrl(cityNode.getUrl());
            }
        }
        List<AddressNode> countryNodeList = new ArrayList<>();
        // 线程池跑country对象..
        for (AddressJsonNode countryNode : countryList) {
            AddressNode addressNode = buildAddressNode(countryNode);
            countryNodeList.add(addressNode);
        }
        return countryNodeList;
    }

    /**
     * @Description: 构建AddressNode对象
     * @Author: liuh
     * @Date: 2020/10/13
     */
    public static AddressNode buildAddressNode(AddressJsonNode addressJsonNode) {
        AddressNode addressNode = new AddressNode();
        addressNode.setLevel(3);
        addressNode.setNodeUrl(addressJsonNode.getUrl());
        addressNode.setAddressName(addressJsonNode.getName());
        addressNode.setAddressCode(addressJsonNode.getCode());
        AddressNode parentNode = new AddressNode();
        parentNode.setAddressCode(addressJsonNode.getCityCode());
        parentNode.setAddressName(addressJsonNode.getCityName());
        parentNode.setNodeUrl(addressJsonNode.getParentUrl());
        addressNode.setParentNode(parentNode);
        return addressNode;
    }

    @AllArgsConstructor
    @Getter
    public enum RdsKeyFileRelationEnum {
        CITY_FILE(AreaConstant.RDS_CITY_KEY, "市.json"),
        COUNTY_FILE(AreaConstant.RDS_COUNTY_KEY, "区县.json"),
        ;
        private String key;
        private String fileName;

        public static String getFileBykey(String key) {
            Optional<RdsKeyFileRelationEnum> optional = Arrays.stream(RdsKeyFileRelationEnum.values()).filter(c -> c.getKey().equals(key)).findFirst();
            if (optional.isPresent() && optional.get() != null) {
                return sourcePath.append(optional.get().getFileName()).toString();
            }
            return "";
        }
    }

    /**
     * @Description: 获取数据..
     * @Author: liuh
     * @Date: 2020/10/12
     */
    public static String getRdsDataByKey(String key) throws Exception {
        Jedis jedis = RedisUtils.getJedisPool().getResource();
        String rdsData = jedis.get(key);
        if (rdsData == null) {
            // 1. 将市信息，存到redis. (FileIuputSteram 大文件读取java.io.FileNotFoundException:  (Result too large))
            StringBuilder sb = new StringBuilder();
            String fileBykey = TownSpiderMain.RdsKeyFileRelationEnum.getFileBykey(AreaConstant.RDS_COUNTY_KEY);
            ReaderListener readerListener = new ReaderListener() {
                @Override
                public void output(List<String> lineList) throws Exception {
                    for (String s : lineList) {
                        sb.append(s);
                    }
                }
            };
            readerListener.setReadColNum(100000);
            NioFileReader reader = new NioFileReader(readerListener, "utf-8");
            try {
                reader.readFileByLine(fileBykey);
            } catch (Exception ex) {
                ex.printStackTrace();
            }
            rdsData = sb.toString();
            if (rdsData == null || rdsData.isEmpty()) {
                throw new Exception("任务取消： 区/县信息加载为空..");
            }
            jedis.set(key, rdsData);
        }
        return rdsData;
    }

    /**
     * @Description: 测试爬取单个..
     * @Author: liuh
     * @Date: 2020/10/12
     */
    public static void testSingleSpiderCounty() {
        ThreadPoolManager instance = ThreadPoolManager.getInstance();
        ExecutorService threadPool = instance.getCachedThreadPool();
        CountDownLatch townCountDown = new CountDownLatch(1);
        AddressNode addressNode = new AddressNode();
        addressNode.setLevel(3);
        addressNode.setNodeUrl("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/13/02/130202.html");
        addressNode.setAddressName("路南区");
        addressNode.setAddressCode("130202000000");
        AddressNode parentNode = new AddressNode();
        parentNode.setAddressCode("130200000000");
        parentNode.setAddressName("唐山市");
        parentNode.setNodeUrl("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/13/1302.html");
        addressNode.setParentNode(parentNode);
//        AreaSpiderCountyTask countyTask = new AreaSpiderCountyTask(addressNode, townCountDown, AreaConstant.HttpUtilEnum.HTTP_CLIENT);
//        threadPool.execute(countyTask);
        try {
            townCountDown.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

}
